﻿using Ivony.Html;
using Ivony.Html.Parser;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using CGT.Infrastructure.Entity;
using CGT.Infrastructure.Service;
using CGT.Infrastructure.Repository;
using System.Threading.Tasks;

namespace ConsoleTest
{
    public class Spiders
    {
        string website = "https://www.cn357.com/";
        public void excute()
        {

            //获取批次url集合
            //var lotList = GetLotUrlList();
            //获取批次内的车型url集合
            //var vehicleList = GetVehicleUrlList(lotList);
            //获取每个车型的规格
            //GetAllVehilceInfo(vehicleList);

            //解析车型url
            GetCarSeries();
        }

        //获取批次的url集合
        public List<CN357_Lot> GetLotUrlList()
        {
            var TextListUrl = "https://www.cn357.com/notice_list";
            IHtmlDocument source = GetProxyHtml(TextListUrl);
            var aList = source.Find(".lotList a");
            var lotList = new List<CN357_Lot>();
            foreach (var i in aList)
            {
                var lot = new CN357_Lot();
                var url = website + i.Attribute("href").Value();
                lot.Url = url;
                lot.Lot = Convert.ToInt32(url.Split('_')[1]);
                lotList.Add(lot);
            }
            var service = new CN357_LotService();
            service.Insert(lotList);
            return lotList;
        }
        //获取批次内的车型url集合
        public List<string> GetVehicleUrlList(List<string> lotList)
        {
            var vehicleUrlList = new List<string>();
            foreach (var lotUrl in lotList)
            {
                IHtmlDocument source = GetProxyHtml(lotUrl);
                var pages = source.Find(".pageList a");//获取产品页的页码对象，  url格式为/notice_批次_页码  ，第一页为 /notice_批次
                //所有页码url集合
                var listPages = new List<string>();
                listPages.Add(lotUrl);//第一页，当前页
                source.Find(".pageList a:last-child").Remove();//‘下一页’的a标签移除掉
                var lastpages = Convert.ToInt32(source.Find(".pageList a:last-child").Single().InnerText());//获取最后一页页码
                //i=1是因为第一页url为 /notice_批次，没有页码后缀
                for (var i = 1; i <= lastpages; i++)
                {
                    var pageUrl = lotUrl + "_" + i.ToString();
                    listPages.Add(pageUrl);
                }
                //遍历所有页码页
                foreach (var pageUrl in listPages)
                {
                    IHtmlDocument pageSource = GetProxyHtml(lotUrl);
                    var first_td = pageSource.Find(".listTable tbody tr:first-child");
                    first_td.Remove();
                    var tds = pageSource.Find(".listTable tbody tr");//tr集合
                    foreach (var td in tds)
                    {
                        //td内如果存在a标签，则是具体车型url连接
                        if (td.Exists("a"))
                        {
                            var a = td.Find("a").First();
                            var url = website + a.Attribute("href").Value();
                            vehicleUrlList.Add(url);
                            Thread.Sleep(2000);
                        }
                    }
                }
            }
            return vehicleUrlList;
        }

        /// <summary>
        /// 解析车型页
        /// </summary>
        /// <returns></returns>
        public void GetCarSeries()
        {
            var tasks = new List<Task>();
            var fileList = Scan(@"C:\cn357CarSeriesLot\276-300");
            var taskSetFileCount = 200;//每个线程处理的文件数
            var tasksCount = (int)Math.Ceiling((double)fileList.Count / taskSetFileCount);//需要开启的线程数
            var list = new List<CN357_CarSeries>();
            for (var i = 0; i < tasksCount; i++)
            {
                tasks.Add(Task.Factory.StartNew((index) =>
                {
                    var indexStart = (int)index * taskSetFileCount;
                    var indexEnd = ((int)index + 1) * taskSetFileCount - 1;
                    var pageIndex = (int)index + 1;
                    var pageSize = taskSetFileCount;
                    var currentList = fileList.Skip((pageIndex - 1) * pageSize).Take(pageSize).ToList();
                    Console.WriteLine(string.Format("Task {0} Start ，Will Process {1}-{2} Lot Data", index, indexStart, indexEnd));
                    foreach (var file in currentList)
                    {
                        //页面html
                        IHtmlDocument pageSource = GetProxyHtml(file.FilePath);
                        var first_td = pageSource.Find("table tr:first-child");
                        first_td.Remove();
                        var trs = pageSource.Find("table tr");//tr集合
                        foreach (var tr in trs)
                        {
                            var series = new CN357_CarSeries();
                            //tr内如果存在a标签，则是具体车型url连接
                            if (tr.Exists("a"))
                            {
                                var a = tr.Find("a").First();
                                var url = website + a.Attribute("href").Value();
                                series.LotId = file.Lot;
                                series.Url = url;
                                series.CarSeries = a.InnerText();
                                list.Add(series);
                            }
                            //每个tr里有3个td
                            var tds = tr.Find("td");
                            var tdIndex = 1;
                            foreach (var td in tds)
                            {
                                //第二个tr里是公司名称
                                if (tdIndex == 2)
                                    series.CompanyName = td.InnerText();
                                tdIndex += 1;
                            }
                        }
                    }
                    Console.WriteLine(string.Format("---Task {0} End", index));
                }, i));
            }
            Task.WaitAll(tasks.ToArray());
            BatchInsert(list);
        }

        /// <summary>
        /// 批量插入
        /// </summary>
        public void BatchInsert(List<CN357_CarSeries> list)
        {
            var service = new CN357_CarSeriesService();
            Console.WriteLine(string.Format("Insert Count Amount {0}", list.Count()));
            //单线程  每次1000条 批量插入
            var singleSetCount = 200;
            //需要批量插入的次数
            var lot = (int)Math.Ceiling((double)list.Count / singleSetCount);
            for (var i = 0; i < lot; i++)
            {

                Console.WriteLine(string.Format("-lot for {0} data Insert Begin-", i));
                var pageIndex = i;
                var pageSize = singleSetCount;
                var currentList = list.Skip((pageIndex - 1) * pageSize).Take(pageSize).ToList();
                try
                {
                    if (i == 83)
                    {
                        var nullCount = 0;
                        for (int j = 0; j < currentList.Count; j++)
                        {
                            if (currentList[j] == null)
                            {
                                nullCount += 1;
                                currentList[j] = new CN357_CarSeries()
                                {
                                    LotId = -1
                                };
                            }
                        }

                        Console.WriteLine(string.Format("-----lot for {0} data nullCount = {1}", i, nullCount));
                        service.Insert(currentList);
                        Console.WriteLine(string.Format("-lot for {0} data Insert End-", i));
                    }

                }
                catch (Exception ex)
                {
                    Console.WriteLine(string.Format("****************Error*******Index={0}****Message={1}", i, ex.Message));
                }
            }
            Console.WriteLine(string.Format("BatchInsert End"));

        }

        public void GetAllVehilceInfo(List<string> vehicleInfoUrlList)
        {
            //遍历，每一条详细信息页面
            foreach (var vehicleInfoUrl in vehicleInfoUrlList)
            {
                IHtmlDocument source = GetProxyHtml(vehicleInfoUrl);

            }
        }
        /// <summary>
        /// 根据文件路径path获取解析后的html模板
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public IHtmlDocument GetProxyHtml(string url)
        {
            //return new JumonyParser().LoadDocument(url, System.Text.Encoding.GetEncoding("utf-8"));

            if (string.IsNullOrWhiteSpace(url))
                url = @"C:\notice_list.txt";
            StreamReader sr = new StreamReader(url);
            var str = "";
            string content;
            while ((content = sr.ReadLine()) != null)
            {
                str += content.ToString();
            }
            return new JumonyParser().Parse(str);
        }

        /// <summary>
        /// 获取目录下所有子文件夹内文件
        /// </summary>
        /// <returns></returns>
        public List<FileModel> Scan(string path = @"C:\cn357CarSeries")
        {
            var filePathList = new List<FileModel>();
            DirectoryInfo theFolder = new DirectoryInfo(path);
            //子文件夹
            foreach (DirectoryInfo NextFolder in theFolder.GetDirectories())
            {
                //遍历每个子文件夹下的文件
                FileInfo[] fileInfo = NextFolder.GetFiles();
                foreach (FileInfo NextFile in fileInfo)
                {
                    var file = new FileModel();
                    file.FileName = NextFile.Name;
                    file.FilePath = NextFile.FullName;
                    file.Lot = Convert.ToInt32(NextFolder.Name);
                    filePathList.Add(file);
                }
            }
            return filePathList;
        }
    }

    //临时类，存放批次和文件名
    public class FileModel
    {
        /// <summary>
        /// 文件名称
        /// </summary>
        public string FileName { get; set; }
        /// <summary>
        /// 文件路径
        /// </summary>
        public string FilePath { get; set; }
        /// <summary>
        /// 批次
        /// </summary>
        public int Lot { get; set; }
    }

}